package com.study.flink.java.day05_state;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.util.Collector;

import java.util.Properties;

/**
 * state是flink灵魂
 * 轻量级的快照中间结果，state存储到外部系统，失败了再从外部程序恢复
 */
public class MapWithState {

    public static void main(String[] args) throws Exception{

        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //5秒钟更新State
        env.enableCheckpointing(5000);

        // 中间结果保存本地
        env.setStateBackend(new FsStateBackend("file:///D:\\IDEA\\flink-study\\dir\\day05\\backend"));

        //实现EXACTLY_ONCE，必须记录偏移量
        env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);

        //把任务停掉之后，依然保存之前的checkpoint
        env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);

        // kafka的partitions有3个，对应有3个source
        Properties props = new Properties();
        props.setProperty("bootstrap.servers", "node02:9092"); //kafka的broker地址
        props.setProperty("group.id", "gid-wc10");//指定组ID
        props.setProperty("auto.offset.reset", "earliest");//没有记录偏移量，第一次从最开始消费
        //不自动提交偏移量，而是交给Flink通过Checkpointing管理偏移量
        props.setProperty("enable.auto.commit", "false");

        // 用kafka的并行source，每一个组都要满足条件才会触发
        FlinkKafkaConsumer<String> wc11 = new FlinkKafkaConsumer<>("wc11", new SimpleStringSchema(), props);
        DataStream<String> lines = env.addSource(wc11);

        SingleOutputStreamOperator<String> words = lines.flatMap(new FlatMapFunction<String, String>() {
            @Override
            public void flatMap(String line, Collector<String> out) throws Exception {
                // 切分 压平
                String[] words = line.split(" ");
                for (String word : words) {
                    out.collect(word);
                }
            }
        });

        SingleOutputStreamOperator<Tuple2<String, Integer>> wordAndCount = words.map(new MapFunction<String, Tuple2<String, Integer>>() {
            @Override
            public Tuple2<String, Integer> map(String s) throws Exception {
                if("error".equals(s)) {
                    throw new RuntimeException("BY hand!");
                }
                // (单词,次数)
                return Tuple2.of(s, 1);
            }
        });

        // 为了保证程序出现问题可以继续累加，要记录分组聚合的中间结果
        KeyedStream<Tuple2<String, Integer>, Tuple> keyed = wordAndCount.keyBy(0);

        SingleOutputStreamOperator<Tuple2<String, Integer>> summed = keyed.map(new RichMapFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>() {

            private transient ValueState<Tuple2<String, Integer>> valueState;

            @Override
            public void open(Configuration parameters) throws Exception {
                //初始化状态或恢复历史状态
                //定义一个状态描述器
                ValueStateDescriptor<Tuple2<String, Integer>> descriptor = new ValueStateDescriptor<>(
                        "wc-keyed-state", //每个应用都有自己的状态
                        //TypeInformation.of(new TypeHint<Tuple2<String, Integer>>() {})
                        Types.TUPLE(Types.STRING, Types.INT)
                );
                valueState = getRuntimeContext().getState(descriptor);
            }

            @Override
            public Tuple2<String, Integer> map(Tuple2<String, Integer> value) throws Exception {

                String word = value.f0;
                Integer count = value.f1;

                Tuple2<String, Integer> historyKv = valueState.value();
                if (historyKv == null) {
                    //第一次更新存放历史数据
                    historyKv = value;
                } else {
                    //累加
                    historyKv.f1 = historyKv.f1 + count;
                }
                valueState.update(historyKv);
                return historyKv;
            }
        });

        summed.print();

        env.execute("MapWithState-java");

    }

}
